import requests
from lxml import etree
import re
from multiprocessing.dummy import Pool

# 需求：爬取梨视频的视频数据
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0'
}
# 原则：线程池处理的是阻塞且较为耗时的操作

# 对下述url发起请求解析出视频详情页的url和视频的名称
url = 'https://www.pearvideo.com/category_5'
page_text = requests.get(url=url, headers=headers).text

tree = etree.HTML(page_text)
li_list = tree.xpath('//ul[@class="listvideoListUl"]/li"]')
urls = []  # 存储所有视频的链接
for li in li_list:
    detail_url = 'https://www.pearvideo.com' + li.xpath('./div/a/@href')[0]
    name = li.xpath('./div/a/div[2]/text()')[0] + '.mp4'
    # print(detail_url, name)
    # 对详情页的url发起请求
    detail_page_text = requests.get(url=detail_url, headers=headers).text
    # 从详情页中解析出视频的地址（url）
    ex = 'srcUrl="(.*?)",vdoUrl'
    video_url = re.findall(ex, detail_page_text)
    # print(video_url)
    dic = {
        'name': name,
        'url': video_url
    }
    urls.append(dic)


def get_video_data(dic):
    url = dic['url']
    print(dic['name', '正在下载.....'])
    data = requests.get(url=url, headers=headers).content
    # 持久化存储操作
    with open(dic['name'], 'wb') as fp:
        fp.write(data)
        print(dic['name', '下载成功！'])


# 使用线程池对视频数据进行请求（较为耗时的阻塞操作）
pool = Pool(4)
pool.map(get_video_data, urls)

pool.close()
pool.join()
